package com.ljy.crawler.util;

import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.springframework.stereotype.Component;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.UUID;

/**
 * @program: WebCrawler
 * @description: 根据url获取html、下载图片
 * @author: liangjiayy
 * @create: 2022-01-25 21:27
 **/
@Component
public class HttpUtils {
    // 连接池
    private PoolingHttpClientConnectionManager cm;

    // 图片存放目录
    public static final String imgLocation = "C:\\Users\\AikeTech\\Desktop\\temp\\jd-img\\";

    public HttpUtils(){
        cm = new PoolingHttpClientConnectionManager();
        // 设置最大连接数
        cm.setMaxTotal(100);
        // 设置每个主机的最大连接数
        cm.setDefaultMaxPerRoute(10);
    }
    // 获取html内容
    public String getHtml(String url){
        // 获取 httpclient 对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
        // 设置请求
        HttpGet httpGet=new HttpGet(url);
        httpGet.setConfig(config());
        //必须设置，否则爬取不了京东，在浏览器 F12 查看即可
        httpGet.setHeader("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36 Edg/97.0.1072.69");
        // 发送请求
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode()==200) {
                if (response.getEntity() != null){
                    return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                if (response != null) {
                    response.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return "";
    }

    // 下载图片
    public  String getImg(String url){
        // 获取 httpclient 对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
        // 设置请求
        url = url.startsWith("//")? "https:" + url : url;
        HttpGet httpGet=new HttpGet(url);
        httpGet.setConfig(config());
        // 发送请求
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode()==200) {
                if (response.getEntity() != null){
                    String imgName = UUID.randomUUID().toString().replaceAll("-","") +
                            url.substring(url.lastIndexOf("."));
                    OutputStream outputStream =
                            new FileOutputStream(imgLocation + imgName);
                    response.getEntity().writeTo(outputStream);
                    return imgName;
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                if (response != null) {
                    response.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return "";
    }

    // 配置http请求
    private RequestConfig config() {
        return RequestConfig.custom()
                .setConnectTimeout(1000 * 20)//创建连接最长时间
                .setConnectionRequestTimeout(1000 * 20)//获取连接最长时间
                .setSocketTimeout(1000 * 30)//数据传输
                // 代理 ip 网站：https://www.kuaidaili.com/free/inha/
                .setProxy(new HttpHost("103.37.141.69", 80))
                .build();
    }
}
